#!/usr/bin/python
# -*- coding:utf-8 -*-
import scrapy
from scrapy import FormRequest,Request
from scrapy.selector import Selector
import sys
import time
import codecs
import os
from tutorial.items import SMTXItem
reload(sys)
sys.setdefaultencoding('utf8')

class miaomuzhanmmpider(scrapy.Spider):
    currentSpiderHaveDone =[]
    name = "miaomuzhan"
    custom_settings = {'ITEM_PIPELINES':{'tutorial.pipelines.miaomuzhanPipeline':900,}}
    allowed_domains = ["miaomuzhan.com"]
    start_urls = []
    GetthisAll =True
    existslist=[]

    if GetthisAll:
        Historyfile = codecs.open(os.getcwd()+"\\"+'miaomuzhan.txt','r',encoding="utf-8")

        s = Historyfile.readlines()
        Historyfile.flush()
        Historyfile.close()
        for fileLine in s:
            existslist.append(fileLine.strip('\n'))


        for num in range(1,826):
        #for num in range(1,2):
            waitingtoscrapy ='http://www.miaomuzhan.com/company/?cname=&nativeplace=0&totalresult=16485&pageno={0}'.format(str(num))
            if waitingtoscrapy not in existslist:
                print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
                print waitingtoscrapy
                print ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
                start_urls.append(waitingtoscrapy)

    def parse(self, response):
        if response.url not in self.existslist:
            Historyfile = codecs.open(os.getcwd()+"\\"+'miaomuzhan.txt','a+',encoding="utf-8")
            Historyfile.writelines(str(response.url)+"\n")
            Historyfile.close()

        strdata =str( response.text.encode('utf-8'))
        sel = Selector(text=strdata)
        trs = sel.xpath('//ul[@class="e2"]/li')
        for tr in trs:
            productaddress = str(tr.xpath('a/@href').extract()[0])
            desc =str(tr.xpath('p/text()').extract()[0])
            cname =str(tr.xpath('a/text()').extract()[0])
            address = str(tr.xpath('text()').extract()[0])
            title = str(tr.xpath('span/text()').extract()[0])
            item = SMTXItem()
            item['address'] =address
            item['title'] = title
            item['desc'] =desc
            item["link"]=productaddress
            item['cname'] =cname
            if productaddress in self.existslist:
                print ">>>>>>>>>>>>>>have done before >>>>>>>>>>>>>>"
                continue
            else:
                time.sleep(1)
                print ">>>>>>>>>>>>>>go to the url : "+productaddress+" >>>>>>>>>>>>>>"
                yield scrapy.Request(productaddress,callback=self.GetDetailInfo,meta={'item':item})

    def GetDetailInfo(self,response):
        if response.url not in self.existslist:
            Historyfile = codecs.open(os.getcwd()+"\\"+'miaomuzhan.txt','a+',encoding="utf-8")
            Historyfile.writelines(str(response.url)+"\n")
            Historyfile.close()
        sel = Selector(response)
        item = response.meta['item']
        item['people'] = ''
        item['QQ'] = ''
        item['tel'] = ''
        item['mobilephone'] = ''
        item['fax'] = ''
        item['email'] = ''
        item['weixin']=''
        dds = sel.xpath('//dl[@class="profile"]/dd')
        print 8888888888888
        print len(dds)
        for dd in  dds:
            text = str(dd.xpath("string(.)").extract()[0])
            print text +"   6666666666666"
            if '联系人' in text:
                item['people'] = text
            elif 'QQ' in text:
                item['QQ'] = text
            elif '电话' in text:
                item['tel'] = text
            elif '手机' in text:
                item['mobilephone'] = text
            elif '传真' in text:
                item['fax'] = text
            elif 'Email' in text:
                item['email'] = text
            elif '微信号' in text:
                item['weixin'] = text
            elif '地址' in text:
                item['address'] = text
        yield item

    def buildItem(self,item,text):
        print ">>>>>>>>>>>>>> building data  >>>>>>>>>>>>>>"
        if '联系人' in text:
            item['people'] = text
        elif 'QQ' in text:
            item['QQ'] = text
        elif '电话' in text:
            item['tel'] = text
        elif '手机' in text:
            item['mobilephone'] = text
        elif '传真' in text:
            item['fax'] = text
        elif 'Email' in text:
            item['email'] = text
        elif '微信号' in text:
            item['weixin'] = text
        elif '地址' in text:
            item['address'] = text
